{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use DQN to Play MoutainCar-v0\n",
    "\n",
    "TensorFlow version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow.compat.v2 as tf\n",
    "tf.random.set_seed(0)\n",
    "from tensorflow import nn\n",
    "from tensorflow import losses\n",
    "from tensorflow import optimizers\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import models\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:21:46 [INFO] env: <MountainCarEnv<MountainCar-v0>>\n",
      "22:21:46 [INFO] action_space: Discrete(3)\n",
      "22:21:46 [INFO] observation_space: Box(-1.2000000476837158, 0.6000000238418579, (2,), float32)\n",
      "22:21:46 [INFO] reward_range: (-inf, inf)\n",
      "22:21:46 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 30}\n",
      "22:21:46 [INFO] _max_episode_steps: 200\n",
      "22:21:46 [INFO] _elapsed_steps: None\n",
      "22:21:46 [INFO] id: MountainCar-v0\n",
      "22:21:46 [INFO] entry_point: gym.envs.classic_control:MountainCarEnv\n",
      "22:21:46 [INFO] reward_threshold: -110.0\n",
      "22:21:46 [INFO] nondeterministic: False\n",
      "22:21:46 [INFO] max_episode_steps: 200\n",
      "22:21:46 [INFO] _kwargs: {}\n",
      "22:21:46 [INFO] _env_name: MountainCar\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('MountainCar-v0')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQNReplayer:\n",
    "    def __init__(self, capacity):\n",
    "        self.memory = pd.DataFrame(index=range(capacity),\n",
    "                columns=['state', 'action', 'reward', 'next_state', 'done'])\n",
    "        self.i = 0\n",
    "        self.count = 0\n",
    "        self.capacity = capacity\n",
    "\n",
    "    def store(self, *args):\n",
    "        self.memory.loc[self.i] = args\n",
    "        self.i = (self.i + 1) % self.capacity\n",
    "        self.count = min(self.count + 1, self.capacity)\n",
    "\n",
    "    def sample(self, size):\n",
    "        indices = np.random.choice(self.count, size=size)\n",
    "        return (np.stack(self.memory.loc[indices, field]) for field in\n",
    "                self.memory.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQNAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        self.replayer = DQNReplayer(10000)\n",
    "\n",
    "        self.evaluate_net = self.build_net(\n",
    "                input_size=env.observation_space.shape[0],\n",
    "                hidden_sizes=[64, 64], output_size=self.action_n)\n",
    "        self.target_net = models.clone_model(self.evaluate_net)\n",
    "\n",
    "    def build_net(self, input_size, hidden_sizes, output_size):\n",
    "        model = keras.Sequential()\n",
    "        for layer, hidden_size in enumerate(hidden_sizes):\n",
    "            kwargs = dict(input_shape=(input_size,)) if not layer else {}\n",
    "            model.add(layers.Dense(units=hidden_size,\n",
    "                    activation=nn.relu, **kwargs))\n",
    "        model.add(layers.Dense(units=output_size))\n",
    "        optimizer = optimizers.Adam(lr=0.001)\n",
    "        model.compile(loss=losses.mse, optimizer=optimizer)\n",
    "        return model\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "            self.target_net.set_weights(self.evaluate_net.get_weights())\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        if self.mode == 'train' and np.random.rand() < 0.001:\n",
    "            # epsilon-greedy policy in train mode\n",
    "            action = np.random.randint(self.action_n)\n",
    "        else:\n",
    "            qs = self.evaluate_net.predict(observation[np.newaxis])\n",
    "            action = np.argmax(qs)\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "            if len(self.trajectory) >= 8:\n",
    "                state, _, _, act, next_state, reward, done, _ = \\\n",
    "                        self.trajectory[-8:]\n",
    "                self.replayer.store(state, act, reward, next_state, done)\n",
    "            if self.replayer.count >= self.replayer.capacity * 0.95:\n",
    "                    # skip first few episodes for speed\n",
    "                self.learn()\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def learn(self):\n",
    "        # replay\n",
    "        states, actions, rewards, next_states, dones = self.replayer.sample(1024)\n",
    "\n",
    "        # train\n",
    "        next_qs = self.target_net.predict(next_states)\n",
    "        next_max_qs = next_qs.max(axis=-1)\n",
    "        us = rewards + self.gamma * (1. - dones) * next_max_qs\n",
    "        targets = self.evaluate_net.predict(states)\n",
    "        targets[np.arange(us.shape[0]), actions] = us\n",
    "        self.evaluate_net.fit(states, targets, verbose=0)\n",
    "\n",
    "\n",
    "agent = DQNAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:21:47 [INFO] ==== train ====\n",
      "22:21:56 [DEBUG] train episode 0: reward = -200.00, steps = 200\n",
      "22:22:08 [DEBUG] train episode 1: reward = -200.00, steps = 200\n",
      "22:22:19 [DEBUG] train episode 2: reward = -200.00, steps = 200\n",
      "22:22:32 [DEBUG] train episode 3: reward = -200.00, steps = 200\n",
      "22:22:44 [DEBUG] train episode 4: reward = -200.00, steps = 200\n",
      "22:22:59 [DEBUG] train episode 5: reward = -200.00, steps = 200\n",
      "22:23:13 [DEBUG] train episode 6: reward = -200.00, steps = 200\n",
      "22:23:27 [DEBUG] train episode 7: reward = -200.00, steps = 200\n",
      "22:23:46 [DEBUG] train episode 8: reward = -200.00, steps = 200\n",
      "22:24:08 [DEBUG] train episode 9: reward = -200.00, steps = 200\n",
      "22:24:25 [DEBUG] train episode 10: reward = -200.00, steps = 200\n",
      "22:24:43 [DEBUG] train episode 11: reward = -200.00, steps = 200\n",
      "22:25:09 [DEBUG] train episode 12: reward = -200.00, steps = 200\n",
      "22:25:27 [DEBUG] train episode 13: reward = -200.00, steps = 200\n",
      "22:25:40 [DEBUG] train episode 14: reward = -200.00, steps = 200\n",
      "22:25:54 [DEBUG] train episode 15: reward = -200.00, steps = 200\n",
      "22:26:08 [DEBUG] train episode 16: reward = -200.00, steps = 200\n",
      "22:26:21 [DEBUG] train episode 17: reward = -200.00, steps = 200\n",
      "22:26:35 [DEBUG] train episode 18: reward = -200.00, steps = 200\n",
      "22:26:49 [DEBUG] train episode 19: reward = -200.00, steps = 200\n",
      "22:27:03 [DEBUG] train episode 20: reward = -200.00, steps = 200\n",
      "22:27:17 [DEBUG] train episode 21: reward = -200.00, steps = 200\n",
      "22:27:32 [DEBUG] train episode 22: reward = -200.00, steps = 200\n",
      "22:27:46 [DEBUG] train episode 23: reward = -200.00, steps = 200\n",
      "22:28:01 [DEBUG] train episode 24: reward = -200.00, steps = 200\n",
      "22:28:15 [DEBUG] train episode 25: reward = -200.00, steps = 200\n",
      "22:28:30 [DEBUG] train episode 26: reward = -200.00, steps = 200\n",
      "22:28:45 [DEBUG] train episode 27: reward = -200.00, steps = 200\n",
      "22:28:59 [DEBUG] train episode 28: reward = -200.00, steps = 200\n",
      "22:29:13 [DEBUG] train episode 29: reward = -200.00, steps = 200\n",
      "22:29:28 [DEBUG] train episode 30: reward = -200.00, steps = 200\n",
      "22:29:41 [DEBUG] train episode 31: reward = -200.00, steps = 200\n",
      "22:29:54 [DEBUG] train episode 32: reward = -200.00, steps = 200\n",
      "22:30:07 [DEBUG] train episode 33: reward = -200.00, steps = 200\n",
      "22:30:20 [DEBUG] train episode 34: reward = -200.00, steps = 200\n",
      "22:30:39 [DEBUG] train episode 35: reward = -200.00, steps = 200\n",
      "22:30:59 [DEBUG] train episode 36: reward = -200.00, steps = 200\n",
      "22:31:13 [DEBUG] train episode 37: reward = -200.00, steps = 200\n",
      "22:31:27 [DEBUG] train episode 38: reward = -200.00, steps = 200\n",
      "22:31:42 [DEBUG] train episode 39: reward = -200.00, steps = 200\n",
      "22:31:56 [DEBUG] train episode 40: reward = -200.00, steps = 200\n",
      "22:32:13 [DEBUG] train episode 41: reward = -200.00, steps = 200\n",
      "22:32:27 [DEBUG] train episode 42: reward = -200.00, steps = 200\n",
      "22:32:41 [DEBUG] train episode 43: reward = -200.00, steps = 200\n",
      "22:32:55 [DEBUG] train episode 44: reward = -200.00, steps = 200\n",
      "22:33:12 [DEBUG] train episode 45: reward = -200.00, steps = 200\n",
      "22:33:26 [DEBUG] train episode 46: reward = -200.00, steps = 200\n",
      "22:33:55 [DEBUG] train episode 47: reward = -200.00, steps = 200\n",
      "22:35:07 [DEBUG] train episode 48: reward = -200.00, steps = 200\n",
      "22:36:30 [DEBUG] train episode 49: reward = -200.00, steps = 200\n",
      "22:38:01 [DEBUG] train episode 50: reward = -200.00, steps = 200\n",
      "22:39:35 [DEBUG] train episode 51: reward = -200.00, steps = 200\n",
      "22:41:23 [DEBUG] train episode 52: reward = -200.00, steps = 200\n",
      "22:43:15 [DEBUG] train episode 53: reward = -200.00, steps = 200\n",
      "22:45:08 [DEBUG] train episode 54: reward = -200.00, steps = 200\n",
      "22:46:58 [DEBUG] train episode 55: reward = -200.00, steps = 200\n",
      "22:48:36 [DEBUG] train episode 56: reward = -200.00, steps = 200\n",
      "22:50:23 [DEBUG] train episode 57: reward = -200.00, steps = 200\n",
      "22:52:06 [DEBUG] train episode 58: reward = -200.00, steps = 200\n",
      "22:53:51 [DEBUG] train episode 59: reward = -200.00, steps = 200\n",
      "22:55:38 [DEBUG] train episode 60: reward = -200.00, steps = 200\n",
      "22:57:21 [DEBUG] train episode 61: reward = -200.00, steps = 200\n",
      "22:59:05 [DEBUG] train episode 62: reward = -200.00, steps = 200\n",
      "23:00:59 [DEBUG] train episode 63: reward = -200.00, steps = 200\n",
      "23:02:56 [DEBUG] train episode 64: reward = -200.00, steps = 200\n",
      "23:04:51 [DEBUG] train episode 65: reward = -200.00, steps = 200\n",
      "23:06:42 [DEBUG] train episode 66: reward = -200.00, steps = 200\n",
      "23:08:28 [DEBUG] train episode 67: reward = -200.00, steps = 200\n",
      "23:10:20 [DEBUG] train episode 68: reward = -200.00, steps = 200\n",
      "23:12:28 [DEBUG] train episode 69: reward = -200.00, steps = 200\n",
      "23:14:53 [DEBUG] train episode 70: reward = -200.00, steps = 200\n",
      "23:17:14 [DEBUG] train episode 71: reward = -200.00, steps = 200\n",
      "23:19:34 [DEBUG] train episode 72: reward = -200.00, steps = 200\n",
      "23:21:58 [DEBUG] train episode 73: reward = -200.00, steps = 200\n",
      "23:24:23 [DEBUG] train episode 74: reward = -200.00, steps = 200\n",
      "23:26:48 [DEBUG] train episode 75: reward = -200.00, steps = 200\n",
      "23:29:10 [DEBUG] train episode 76: reward = -200.00, steps = 200\n",
      "23:31:10 [DEBUG] train episode 77: reward = -162.00, steps = 162\n",
      "23:33:30 [DEBUG] train episode 78: reward = -200.00, steps = 200\n",
      "23:35:51 [DEBUG] train episode 79: reward = -200.00, steps = 200\n",
      "23:38:18 [DEBUG] train episode 80: reward = -200.00, steps = 200\n",
      "23:40:18 [DEBUG] train episode 81: reward = -167.00, steps = 167\n",
      "23:42:49 [DEBUG] train episode 82: reward = -200.00, steps = 200\n",
      "23:45:21 [DEBUG] train episode 83: reward = -200.00, steps = 200\n",
      "23:48:03 [DEBUG] train episode 84: reward = -200.00, steps = 200\n",
      "23:50:39 [DEBUG] train episode 85: reward = -200.00, steps = 200\n",
      "23:52:20 [DEBUG] train episode 86: reward = -129.00, steps = 129\n",
      "23:54:52 [DEBUG] train episode 87: reward = -200.00, steps = 200\n",
      "23:57:11 [DEBUG] train episode 88: reward = -200.00, steps = 200\n",
      "23:58:37 [DEBUG] train episode 89: reward = -120.00, steps = 120\n",
      "00:00:07 [DEBUG] train episode 90: reward = -128.00, steps = 128\n",
      "00:01:38 [DEBUG] train episode 91: reward = -129.00, steps = 129\n",
      "00:03:01 [DEBUG] train episode 92: reward = -116.00, steps = 116\n",
      "00:04:32 [DEBUG] train episode 93: reward = -128.00, steps = 128\n",
      "00:06:50 [DEBUG] train episode 94: reward = -200.00, steps = 200\n",
      "00:09:11 [DEBUG] train episode 95: reward = -200.00, steps = 200\n",
      "00:11:32 [DEBUG] train episode 96: reward = -200.00, steps = 200\n",
      "00:13:51 [DEBUG] train episode 97: reward = -200.00, steps = 200\n",
      "00:16:10 [DEBUG] train episode 98: reward = -200.00, steps = 200\n",
      "00:17:32 [DEBUG] train episode 99: reward = -117.00, steps = 117\n",
      "00:18:56 [DEBUG] train episode 100: reward = -119.00, steps = 119\n",
      "00:21:15 [DEBUG] train episode 101: reward = -200.00, steps = 200\n",
      "00:23:36 [DEBUG] train episode 102: reward = -200.00, steps = 200\n",
      "00:25:56 [DEBUG] train episode 103: reward = -200.00, steps = 200\n",
      "00:28:19 [DEBUG] train episode 104: reward = -200.00, steps = 200\n",
      "00:30:38 [DEBUG] train episode 105: reward = -200.00, steps = 200\n",
      "00:33:03 [DEBUG] train episode 106: reward = -200.00, steps = 200\n",
      "00:34:32 [DEBUG] train episode 107: reward = -129.00, steps = 129\n",
      "00:36:47 [DEBUG] train episode 108: reward = -193.00, steps = 193\n",
      "00:39:10 [DEBUG] train episode 109: reward = -200.00, steps = 200\n",
      "00:41:29 [DEBUG] train episode 110: reward = -200.00, steps = 200\n",
      "00:43:49 [DEBUG] train episode 111: reward = -200.00, steps = 200\n",
      "00:46:10 [DEBUG] train episode 112: reward = -200.00, steps = 200\n",
      "00:48:28 [DEBUG] train episode 113: reward = -200.00, steps = 200\n",
      "00:50:47 [DEBUG] train episode 114: reward = -200.00, steps = 200\n",
      "00:53:08 [DEBUG] train episode 115: reward = -200.00, steps = 200\n",
      "00:55:27 [DEBUG] train episode 116: reward = -200.00, steps = 200\n",
      "00:57:47 [DEBUG] train episode 117: reward = -200.00, steps = 200\n",
      "00:59:45 [DEBUG] train episode 118: reward = -170.00, steps = 170\n",
      "01:02:04 [DEBUG] train episode 119: reward = -200.00, steps = 200\n",
      "01:04:22 [DEBUG] train episode 120: reward = -198.00, steps = 198\n",
      "01:06:03 [DEBUG] train episode 121: reward = -147.00, steps = 147\n",
      "01:08:22 [DEBUG] train episode 122: reward = -200.00, steps = 200\n",
      "01:10:41 [DEBUG] train episode 123: reward = -200.00, steps = 200\n",
      "01:12:59 [DEBUG] train episode 124: reward = -200.00, steps = 200\n",
      "01:15:19 [DEBUG] train episode 125: reward = -200.00, steps = 200\n",
      "01:17:37 [DEBUG] train episode 126: reward = -200.00, steps = 200\n",
      "01:19:56 [DEBUG] train episode 127: reward = -200.00, steps = 200\n",
      "01:22:17 [DEBUG] train episode 128: reward = -200.00, steps = 200\n",
      "01:24:36 [DEBUG] train episode 129: reward = -200.00, steps = 200\n",
      "01:26:25 [DEBUG] train episode 130: reward = -156.00, steps = 156\n",
      "01:28:12 [DEBUG] train episode 131: reward = -148.00, steps = 148\n",
      "01:29:52 [DEBUG] train episode 132: reward = -142.00, steps = 142\n",
      "01:32:04 [DEBUG] train episode 133: reward = -189.00, steps = 189\n",
      "01:33:49 [DEBUG] train episode 134: reward = -149.00, steps = 149\n",
      "01:35:31 [DEBUG] train episode 135: reward = -147.00, steps = 147\n",
      "01:37:09 [DEBUG] train episode 136: reward = -140.00, steps = 140\n",
      "01:38:51 [DEBUG] train episode 137: reward = -145.00, steps = 145\n",
      "01:40:32 [DEBUG] train episode 138: reward = -143.00, steps = 143\n",
      "01:42:22 [DEBUG] train episode 139: reward = -157.00, steps = 157\n",
      "01:43:32 [DEBUG] train episode 140: reward = -99.00, steps = 99\n",
      "01:44:43 [DEBUG] train episode 141: reward = -102.00, steps = 102\n",
      "01:46:00 [DEBUG] train episode 142: reward = -111.00, steps = 111\n",
      "01:48:17 [DEBUG] train episode 143: reward = -200.00, steps = 200\n",
      "01:50:11 [DEBUG] train episode 144: reward = -162.00, steps = 162\n",
      "01:52:31 [DEBUG] train episode 145: reward = -200.00, steps = 200\n",
      "01:54:23 [DEBUG] train episode 146: reward = -159.00, steps = 159\n",
      "01:55:25 [DEBUG] train episode 147: reward = -87.00, steps = 87\n",
      "01:56:33 [DEBUG] train episode 148: reward = -96.00, steps = 96\n",
      "01:57:34 [DEBUG] train episode 149: reward = -86.00, steps = 86\n",
      "01:59:20 [DEBUG] train episode 150: reward = -153.00, steps = 153\n",
      "02:01:03 [DEBUG] train episode 151: reward = -147.00, steps = 147\n",
      "02:03:22 [DEBUG] train episode 152: reward = -200.00, steps = 200\n",
      "02:05:40 [DEBUG] train episode 153: reward = -200.00, steps = 200\n",
      "02:07:31 [DEBUG] train episode 154: reward = -158.00, steps = 158\n",
      "02:09:25 [DEBUG] train episode 155: reward = -162.00, steps = 162\n",
      "02:10:30 [DEBUG] train episode 156: reward = -92.00, steps = 92\n",
      "02:11:55 [DEBUG] train episode 157: reward = -122.00, steps = 122\n",
      "02:13:07 [DEBUG] train episode 158: reward = -103.00, steps = 103\n",
      "02:15:07 [DEBUG] train episode 159: reward = -171.00, steps = 171\n",
      "02:17:25 [DEBUG] train episode 160: reward = -200.00, steps = 200\n",
      "02:18:24 [DEBUG] train episode 161: reward = -84.00, steps = 84\n",
      "02:20:24 [DEBUG] train episode 162: reward = -172.00, steps = 172\n",
      "02:21:25 [DEBUG] train episode 163: reward = -87.00, steps = 87\n",
      "02:22:53 [DEBUG] train episode 164: reward = -125.00, steps = 125\n",
      "02:24:17 [DEBUG] train episode 165: reward = -118.00, steps = 118\n",
      "02:26:10 [DEBUG] train episode 166: reward = -162.00, steps = 162\n",
      "02:27:15 [DEBUG] train episode 167: reward = -89.00, steps = 89\n",
      "02:28:28 [DEBUG] train episode 168: reward = -105.00, steps = 105\n",
      "02:30:15 [DEBUG] train episode 169: reward = -154.00, steps = 154\n",
      "02:32:05 [DEBUG] train episode 170: reward = -158.00, steps = 158\n",
      "02:33:53 [DEBUG] train episode 171: reward = -155.00, steps = 155\n",
      "02:34:51 [DEBUG] train episode 172: reward = -83.00, steps = 83\n",
      "02:35:58 [DEBUG] train episode 173: reward = -97.00, steps = 97\n",
      "02:37:02 [DEBUG] train episode 174: reward = -91.00, steps = 91\n",
      "02:38:55 [DEBUG] train episode 175: reward = -160.00, steps = 160\n",
      "02:40:44 [DEBUG] train episode 176: reward = -149.00, steps = 149\n",
      "02:41:44 [DEBUG] train episode 177: reward = -85.00, steps = 85\n",
      "02:43:31 [DEBUG] train episode 178: reward = -156.00, steps = 156\n",
      "02:44:54 [DEBUG] train episode 179: reward = -117.00, steps = 117\n",
      "02:46:04 [DEBUG] train episode 180: reward = -101.00, steps = 101\n",
      "02:47:07 [DEBUG] train episode 181: reward = -91.00, steps = 91\n",
      "02:48:28 [DEBUG] train episode 182: reward = -115.00, steps = 115\n",
      "02:49:46 [DEBUG] train episode 183: reward = -112.00, steps = 112\n",
      "02:51:03 [DEBUG] train episode 184: reward = -111.00, steps = 111\n",
      "02:52:23 [DEBUG] train episode 185: reward = -112.00, steps = 112\n",
      "02:53:39 [DEBUG] train episode 186: reward = -110.00, steps = 110\n",
      "02:54:59 [DEBUG] train episode 187: reward = -112.00, steps = 112\n",
      "02:56:16 [DEBUG] train episode 188: reward = -110.00, steps = 110\n",
      "02:56:16 [INFO] ==== test ====\n",
      "02:56:30 [DEBUG] test episode 0: reward = -109.00, steps = 109\n",
      "02:56:44 [DEBUG] test episode 1: reward = -111.00, steps = 111\n",
      "02:56:58 [DEBUG] test episode 2: reward = -112.00, steps = 112\n",
      "02:57:12 [DEBUG] test episode 3: reward = -109.00, steps = 109\n",
      "02:57:26 [DEBUG] test episode 4: reward = -112.00, steps = 112\n",
      "02:57:40 [DEBUG] test episode 5: reward = -112.00, steps = 112\n",
      "02:57:53 [DEBUG] test episode 6: reward = -109.00, steps = 109\n",
      "02:58:07 [DEBUG] test episode 7: reward = -110.00, steps = 110\n",
      "02:58:21 [DEBUG] test episode 8: reward = -114.00, steps = 114\n",
      "02:58:34 [DEBUG] test episode 9: reward = -112.00, steps = 112\n",
      "02:58:48 [DEBUG] test episode 10: reward = -112.00, steps = 112\n",
      "02:59:02 [DEBUG] test episode 11: reward = -114.00, steps = 114\n",
      "02:59:16 [DEBUG] test episode 12: reward = -111.00, steps = 111\n",
      "02:59:30 [DEBUG] test episode 13: reward = -111.00, steps = 111\n",
      "02:59:43 [DEBUG] test episode 14: reward = -109.00, steps = 109\n",
      "02:59:56 [DEBUG] test episode 15: reward = -112.00, steps = 112\n",
      "03:00:10 [DEBUG] test episode 16: reward = -109.00, steps = 109\n",
      "03:00:24 [DEBUG] test episode 17: reward = -112.00, steps = 112\n",
      "03:00:38 [DEBUG] test episode 18: reward = -109.00, steps = 109\n",
      "03:00:51 [DEBUG] test episode 19: reward = -112.00, steps = 112\n",
      "03:01:05 [DEBUG] test episode 20: reward = -109.00, steps = 109\n",
      "03:01:18 [DEBUG] test episode 21: reward = -111.00, steps = 111\n",
      "03:01:32 [DEBUG] test episode 22: reward = -112.00, steps = 112\n",
      "03:01:46 [DEBUG] test episode 23: reward = -112.00, steps = 112\n",
      "03:02:00 [DEBUG] test episode 24: reward = -112.00, steps = 112\n",
      "03:02:14 [DEBUG] test episode 25: reward = -112.00, steps = 112\n",
      "03:02:28 [DEBUG] test episode 26: reward = -109.00, steps = 109\n",
      "03:02:41 [DEBUG] test episode 27: reward = -109.00, steps = 109\n",
      "03:02:54 [DEBUG] test episode 28: reward = -110.00, steps = 110\n",
      "03:03:08 [DEBUG] test episode 29: reward = -112.00, steps = 112\n",
      "03:03:22 [DEBUG] test episode 30: reward = -109.00, steps = 109\n",
      "03:03:34 [DEBUG] test episode 31: reward = -109.00, steps = 109\n",
      "03:03:48 [DEBUG] test episode 32: reward = -109.00, steps = 109\n",
      "03:04:02 [DEBUG] test episode 33: reward = -109.00, steps = 109\n",
      "03:04:15 [DEBUG] test episode 34: reward = -110.00, steps = 110\n",
      "03:04:29 [DEBUG] test episode 35: reward = -112.00, steps = 112\n",
      "03:04:43 [DEBUG] test episode 36: reward = -109.00, steps = 109\n",
      "03:04:57 [DEBUG] test episode 37: reward = -111.00, steps = 111\n",
      "03:05:10 [DEBUG] test episode 38: reward = -109.00, steps = 109\n",
      "03:05:24 [DEBUG] test episode 39: reward = -111.00, steps = 111\n",
      "03:05:37 [DEBUG] test episode 40: reward = -112.00, steps = 112\n",
      "03:05:52 [DEBUG] test episode 41: reward = -112.00, steps = 112\n",
      "03:06:05 [DEBUG] test episode 42: reward = -112.00, steps = 112\n",
      "03:06:19 [DEBUG] test episode 43: reward = -112.00, steps = 112\n",
      "03:06:33 [DEBUG] test episode 44: reward = -112.00, steps = 112\n",
      "03:06:47 [DEBUG] test episode 45: reward = -109.00, steps = 109\n",
      "03:07:00 [DEBUG] test episode 46: reward = -109.00, steps = 109\n",
      "03:07:14 [DEBUG] test episode 47: reward = -111.00, steps = 111\n",
      "03:07:27 [DEBUG] test episode 48: reward = -109.00, steps = 109\n",
      "03:07:41 [DEBUG] test episode 49: reward = -109.00, steps = 109\n",
      "03:07:55 [DEBUG] test episode 50: reward = -109.00, steps = 109\n",
      "03:08:09 [DEBUG] test episode 51: reward = -112.00, steps = 112\n",
      "03:08:23 [DEBUG] test episode 52: reward = -112.00, steps = 112\n",
      "03:08:36 [DEBUG] test episode 53: reward = -111.00, steps = 111\n",
      "03:08:49 [DEBUG] test episode 54: reward = -109.00, steps = 109\n",
      "03:09:04 [DEBUG] test episode 55: reward = -112.00, steps = 112\n",
      "03:09:17 [DEBUG] test episode 56: reward = -109.00, steps = 109\n",
      "03:09:31 [DEBUG] test episode 57: reward = -112.00, steps = 112\n",
      "03:09:45 [DEBUG] test episode 58: reward = -112.00, steps = 112\n",
      "03:09:59 [DEBUG] test episode 59: reward = -109.00, steps = 109\n",
      "03:10:13 [DEBUG] test episode 60: reward = -109.00, steps = 109\n",
      "03:10:27 [DEBUG] test episode 61: reward = -112.00, steps = 112\n",
      "03:10:40 [DEBUG] test episode 62: reward = -109.00, steps = 109\n",
      "03:10:54 [DEBUG] test episode 63: reward = -109.00, steps = 109\n",
      "03:11:07 [DEBUG] test episode 64: reward = -110.00, steps = 110\n",
      "03:11:20 [DEBUG] test episode 65: reward = -109.00, steps = 109\n",
      "03:11:34 [DEBUG] test episode 66: reward = -109.00, steps = 109\n",
      "03:11:47 [DEBUG] test episode 67: reward = -109.00, steps = 109\n",
      "03:12:00 [DEBUG] test episode 68: reward = -109.00, steps = 109\n",
      "03:12:14 [DEBUG] test episode 69: reward = -112.00, steps = 112\n",
      "03:12:28 [DEBUG] test episode 70: reward = -112.00, steps = 112\n",
      "03:12:42 [DEBUG] test episode 71: reward = -109.00, steps = 109\n",
      "03:12:56 [DEBUG] test episode 72: reward = -112.00, steps = 112\n",
      "03:13:09 [DEBUG] test episode 73: reward = -112.00, steps = 112\n",
      "03:13:23 [DEBUG] test episode 74: reward = -110.00, steps = 110\n",
      "03:13:36 [DEBUG] test episode 75: reward = -109.00, steps = 109\n",
      "03:13:50 [DEBUG] test episode 76: reward = -110.00, steps = 110\n",
      "03:14:04 [DEBUG] test episode 77: reward = -110.00, steps = 110\n",
      "03:14:17 [DEBUG] test episode 78: reward = -112.00, steps = 112\n",
      "03:14:31 [DEBUG] test episode 79: reward = -111.00, steps = 111\n",
      "03:14:45 [DEBUG] test episode 80: reward = -110.00, steps = 110\n",
      "03:14:58 [DEBUG] test episode 81: reward = -112.00, steps = 112\n",
      "03:15:12 [DEBUG] test episode 82: reward = -112.00, steps = 112\n",
      "03:15:26 [DEBUG] test episode 83: reward = -112.00, steps = 112\n",
      "03:15:40 [DEBUG] test episode 84: reward = -112.00, steps = 112\n",
      "03:15:53 [DEBUG] test episode 85: reward = -112.00, steps = 112\n",
      "03:16:07 [DEBUG] test episode 86: reward = -110.00, steps = 110\n",
      "03:16:21 [DEBUG] test episode 87: reward = -112.00, steps = 112\n",
      "03:16:35 [DEBUG] test episode 88: reward = -111.00, steps = 111\n",
      "03:16:48 [DEBUG] test episode 89: reward = -112.00, steps = 112\n",
      "03:17:01 [DEBUG] test episode 90: reward = -109.00, steps = 109\n",
      "03:17:15 [DEBUG] test episode 91: reward = -112.00, steps = 112\n",
      "03:17:29 [DEBUG] test episode 92: reward = -109.00, steps = 109\n",
      "03:17:43 [DEBUG] test episode 93: reward = -114.00, steps = 114\n",
      "03:17:57 [DEBUG] test episode 94: reward = -112.00, steps = 112\n",
      "03:18:11 [DEBUG] test episode 95: reward = -112.00, steps = 112\n",
      "03:18:24 [DEBUG] test episode 96: reward = -113.00, steps = 113\n",
      "03:18:38 [DEBUG] test episode 97: reward = -111.00, steps = 111\n",
      "03:18:51 [DEBUG] test episode 98: reward = -109.00, steps = 109\n",
      "03:19:05 [DEBUG] test episode 99: reward = -110.00, steps = 110\n",
      "03:19:05 [INFO] average episode reward = -110.71 ± 1.46\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABOq0lEQVR4nO29e7QkV3Xf/91V1X1fc+epkUbMaDQSksASOCBdCXAA8RAg84uNgGAU5xeRn+MIY/JbOE5izFIedn4mv/gR2SE28pLxA9nEGGwrIsZCICc8bEsWIxCSRkJo9GQekkYzmrkzcx/dVXXyR51TderUOfXovq1bt+/+rHXX7a6urj5dd2bXt757n31ICAGGYRhmfeGt9gAYhmGYFx8O/gzDMOsQDv4MwzDrEA7+DMMw6xAO/gzDMOuQYLUHUJczzjhD7NmzZ7WHwTAMs6a49957nxdCbDe3r5ngv2fPHuzdu3e1h8EwDLOmIKKnbNtHZvsQ0auI6G4iuo+I9hLRFdprHyOi/UT0CBG9Y1RjYBiGYeyMUvn/CoBfFELcTkTvlM/fREQXA7gWwCUAXgLgTiK6SAgRjXAsDMMwjMYoE74CwEb5eBOAQ/LxuwB8VgixLIR4AsB+AFdY3s8wDMOMiFEq/58BcAcR/RqSi8wPye07Adyt7XdAbitARNcDuB4Adu/ePbKBMgzDrDeGCv5EdCeAHZaXbgDwVgD/UgjxZ0T0YwB+F8BVAMiyv7XBkBDiZgA3A8Dc3Bw3IWIYhlkhhgr+QoirXK8R0S0APiKffh7Ap+TjAwDO0XbdhcwSYhiGYV4ERun5HwJwpXz8FgCPysdfAHAtEU0Q0XkALgRwzwjHwTAMwxiM0vP/5wD+KxEFAJYgvXshxD4i+hyAhwCEAD7MlT4Mw6wlTi2H+MpDz+Ddr9612kMZmJEFfyHEXwO4zPHaxwF8fFSfzTAMM0q+vO8Z/OznvoNLd2/BudtmVns4A8G9fRiGWTcs9EJcf8teHDq+ONRxemEMADh2urcSw1oVOPgzDLNueOL50/jyQ8/iO98/PtRxwjgpPjyx2Hfuoy4QbYWDP8Mw6wa1am005PK1sSgP/t99Zh4X//sv4emjC0N9zijh4M8wTKs5eHwR3376hRU5lgraUTxc8I8qlP+h44sIY4FnTy4N9TmjhIM/wzCt5je+8j38v3/87RU5lor58ZDKXwX/4wv24B9GK3ORGSUc/BmGaR2/99dP4EsPPgMAOHxiCUv9lfHPM+WfPP+vdz6Kux8/2vg4VcFfvc7Bn2EYpgG33PUk/viepwEAR04uI4pXJvgLGfzVReDmrz+WXmSaUJXwjVbIXholHPwZhmkdkRB4dj7xy587ubRiQTS1fZQyFwK9qPmFJU6Dv73Uk5U/wzDMAMQx8Mz8EnphjBcW+isWRCMt6KvPsZVknloOcf+B4+7jVFT7sOfPMAwzAGEc4/hCHwflZKxwxZS/CvrZRaBvUf6f3/t9vPemv8VS3955pq7nv1LjHgUc/BmGaR0qHj9w8ASA4atzFGmdv2bLWJX/Uoh+JNyefoXnr4L+So17FHDwZximdagE7wPSellx5S8y9W9T/srWOblUHvyPOy8OcW6/NsLBn2GY1qGC5v0HEuUvtGA9DHqdvwrwyxblrz5rfiksHV8vjPHs/BLe8mtfzU1EK0v4fvRP78d/+fIjg3+JFYKDP8MwrUPFzH2H5tNtdVoyCCHw3Wfmna/rM3yjEuWv7jTmK0o5AeAbjz6Px58/jf3PnSq83xb8/3r/87hvyN5CKwEHf4ZhWkcobZNTy5nyrmOhfOvp47j6N76Bhw/bLwCqzj8SIvX/bZ5/ZvuUK38AuPepFwrbXMpfCIEjJ5ex0Fv9JUw4+DMM0zpsc7rq+P5Kqb/gaLWsjhvHme3Tj4rHVbZPneD/LRX8tbuBVPkbdysnFvvoRTEWOfgzDMMUCS3Rv47yV0HX5uMDWn2/yPv2ruPMVyR8AeB7z50sbHOVeh45uQwAzhLSFxMO/gzDtAohBGxxvk7wV/u4gqvQPP+yap9M+VcHfyXuw6io/M0k9XMy+C9y8GcYhsmj4iVR8nt2Illt1nY3YBJVKH9btY+tvUOl5y8ENk11jGNngT6uUP7s+TMMwxioIH/W7CQAYMem5Hed3m7qvcuhPbjGFuVvTfhWVfvEApunO/A90j67jvJP+hWx8mcYhjFQQX7nlikAWfCvo/zjktr95PXkdyT0hK87+JclfH2PsHEyKLwneRzLMRvBfz5R/r0wXvUJYBz8GYZpFSrIv+qczbjgzA24dPcWADUTvtJ3X3b0/1eev6iZ8C0N/kTYPN1Fx6fC+FztHY6cWk4fr3bSl4M/wzCtIlX+m6dw589eifO3zwCoF/wz5V/H9km2lZV6llX7+B5h01QHL92+oTA+V52/Uv7A6ls/HPwZhmkVSvkrP139blLq6Vr5SwX8KBbphaAXxekdgUJdD1zKPxZJ8P/wmy/Av3r7y+B7ZFX+hYSvpvxXu9Y/qN6FYRjmxUN58Z4K/rLsp84kL6XYq5S/Xu0DJOq/G2TJW+XZuxK+YSwQeIS3XXxWOsbIUu1TSPjOL2HbTBdHT/fWtvInovcR0T4iioloznjtY0S0n4geIaJ3aNsvI6IH5GufICIqHplhmHFCCIHb7jtYy+dW6jwYQvm7E75Z8NcDs1nuqT7rVC+0NpSLYpFenNQYq5T/Uj/C/FKI3dumAay+8h/W9nkQwHsAfF3fSEQXA7gWwCUArgbwSSLy5cs3AbgewIXy5+ohx8AwTMt59LlT+Mhn70vX5S0jtX2kLgwsCVUXaZ2/y/ZR1T5xvvVCPzSDf/JbiOQCYPscn/LBX5/kFVkSvqrG/9ytMvivZeUvhHhYCGHrTfouAJ8VQiwLIZ4AsB/AFUR0NoCNQoi7RGKy3QLgmmHGwDBM+1GKv85i6Ur5K8XvNbB9orq2j9bVEyiWe+oLxtusH5XwVfge5QJ9qvy1C4Ly+3dvSxLYa135u9gJ4Pva8wNy20752NxuhYiuJ6K9RLT3yJEjIxkowzCjRwXDbz55DEe1pKd933zCN/CSMFVnVayyHv3JMbL99GkD5v56AZAt6asSvgrfo9w8BHXx0MesKn12rxXlT0R3EtGDlp93lb3Nsk2UbLcihLhZCDEnhJjbvn171VAZhmkpmQ0C3Pnws6X7xmbCV/4OLSWZhc+J6vX20bt6AkXlr/v8tuAfWpS/fgg1Vv2CoJT/uS3x/CurfYQQVw1w3AMAztGe7wJwSG7fZdnOMMwYowfuLz34DN5/+W7nvtEQCd9K5a958VFJwjeMY0x2PCz1Y6vtE5vBnyhnFWXzCbL3qOPs2JjMWG698h+QLwC4logmiOg8JInde4QQhwGcJKLXyiqf6wDcNqIxMAzTEpQCvvjsjbj78WOFunrbvsrrT4N/HdunZmO3SOQtmX6YP3YcA1umuwCAk8vF4K9KPRUF5W8p9Zxf6qMbeNg8nTSEW9MzfIno3UR0AMDrAHyRiO4AACHEPgCfA/AQgC8B+LAQQn3TDwH4FJIk8GMAbh9mDAzDtB8VDM/bPoPFfoT5RfvkKaBY6hmkyr9JV88IQgj8ype+i+89ezI7tmb75Es984E4jOO0a6fN9olikV6cABX849zryXHy9tHGyQCTnaTwcbU7ew41yUsIcSuAWx2vfRzAxy3b9wJ4xTCfyzDM2kJ58btks7bD84vYNN2x7uua4VvL89dKPeeXQnzyq49hZiLARWfNAjDW8NWUf89Q/pFAqtDrVPsEHuWSxGqs+t3F/GIfs5MddHwPHZ/G1vZhGIZJUQp41+Yk+D9zYsm5rwqYZvCvVe2j2T6n5fq/usJ3VfuYnn8cC0x3A0wEnl35W6p96ip/AJjq+Kue8OXgzzDMyFFqXrVpfnbeHfwjo85f2T516vxDzfZRwT/MBX/V1VOUTvIKpa0zO9nBvK3U01rto9f5x+l+ipNLifIHgKmuv7Y9f4ZhmDqowPiSVPm7a/3NhK/XoNpHBfelfoxTSvlrQV49jAzP31bq6XvA7GSQHic/xvLgrywgvdRzfinErKb8V9vz5+DPMMzIUR74VMfHtpkunplfdO6bJnx9M+HbVPlHuW3JsaXnL1Ba6hkJgcDzEHhUuCtQxym0d7As5qIf9uRSHxul8p/s+Oz5Mwwz/qhAG/gedmyaLPX8XaWezbp6asrf4vmbk7zMBV1U47bA96wriFUpf3Wx0/MAJ3Xlz7YPwzDrARW4A4+wY+Mknpl32z7Kphmmq6cQwPGFXm6bfmzT9rF19fQJ6PhkX+zFTPiSYftodxhAYist9KLU85/ucsKXYZh1gF6+edamydKEr1LNgyzmogf0o6d7hfel7R2EgH64YldPAV/aPjblb2vsZl/JK3nvKZk03jilVfuw8mcYZtxRAV0p/2One07bI+3to1o6y8ZuTZQ/AByzBP/U9jGqfazK30tsKpvyr7R9jGUcVbnorO75s/JnGGbcMT1/IL+ebX5fyH2br+SlB3QV/HOJWIftYwZ4Vcff8QlhVC/hq3922shOvlWtBaxX+7DyZxhm7DE9fwB4xmH9FBK+8iJgW1HLJNKC+POyi2Zs8/yNah+zF5Cq4w88z3rRMZV/4LB91HdRwX+j7vnXCP5hFFsvPisBB3+GYUaOCmC+R6nydwV/M+HbZJJXlfJXLxcmeRW6eibKvnbC12X7yE2Z7ZMo/8mShO+NX34E/+5/PAgA+PNvH8Rbb/xaaY5kUHgBd4ZhRo6u/M9Syv+EvdbfTPiqO4Amjd0A3fPXWi3HDtvHqvy9RPlblHd1wjfO/Vb9gZTyn+r4WA7j9DjHTvcwOxng2fklfPKrjyGMBX5s7hzc9NXHsGEiwJmzE5XfvSkc/BmGGTkqyBGR7GzppWvampiLuWSTvOp9jiKt9tGEe9rbx6zztyl/L8k7mHccQlYKFVfysiV8k+cnLdU+QNLW+aHD87jud+/BK3duwp4zpkEEzHR9fPAP9+LQiSV88h9fCtLyCysFB3+GYUaO7pETEbqOKhqguJiL5xGI6it/osTeURO3bIusxCJT/h5Z1vAVySSvju9Z1veVdya5hK+Xzy0YpZ4q+G+YSELudDcJ/nufegEf/sy3sHm6g71PHcM9Tx7D++fOwdYNXdz01cfw0u0zuPqSHZXfexA4+DMMM3KiOM4tfuIZC56b+wLI98unogK3v1dguuPjtOan2+r8ozhbyUtZMDqxXKwl8KjQSlqNw9OVP8Gh/JPf80t9THd9BH6SZlU9/f/z7d/FZMfDn//0D+Fv9x/Ff/tfj+Kn3/xSzEwE+Iv7D+HfvONluc9ZSTj4MwwzcvpR+YxYnbQs1PTUa7Z0np4InMFfPRQiezzZ8Qt3ISrha2vvYCakk/F5jkleqs6/nyZ7gaS9AwA8fHge//SH9uDsTVN472W78N7LslVuv/Fzb6n8vsPA1T4Mw4ycKBa1lb9NWQce5co4nZ8jBGZkYDWPp15Xv9XnT3Z89MLsYhFrn2+r9kltn1zwz7YLIbRqn2ySl0r2ApnnDwDvGJGtUwUHf4ZhRk4Yi9TyAMqVv01Ze1492yeMk0VYdKpsn8lOPv8QaZ/f8YvVPvbg76Xv04epT/KyKf+tM11cvmdL5fcaBRz8GYYZOQXPn/JBMr9v8tucRFVnJa9kBa688s/ZPnG2X5RT/sVVuJKunoR+3Ez56zZRqCV8Zy3K/20/cFbuovhiwsGfYZiRExqev+eRc8auNeHrmGlb+Bzp+eePZ5vhm1X7THb8XKmnXs3TsdT524J/oHn+eopAL/XcOJUF/51bpjA7GeQ8/hcbTvgyTEN+63/vx8Vnb8SbX37mag9lzRAann9ZAtcs9Uz2Ry3PP44Fpjp5TWtv7JZ9zmTHy62qpcblS+Ufi+S46YpiIrs45L6PRfnrk7x02+fM2Unc/x/ePpL6/bqw8meYhtz89cfx598+WHv/Oj1pxp2ogeefKn9TWdewfcI4RuB76Aaetq3o+edsn8CwfbQZxh055r5lcXbPvJgZFT76Y30hF8VqBn6Agz/DNCKKBeaX+jh22r0Yic7v/80TeOuNX1v1VZtWm9Dw/NVELBvJEor5wGi2T3ARi+TCMiGDv2/kCkzbxyOgG+QncukJ37SvUFQM6IU7mVT5J7+7QWIFLfUj9KI4V+3TBjj4M0wDTiz2IQRw7HS/1v5PHV3AE8+fxuf3fn/EI2s3Vb1wdELNYtH3r+f5JxcZNYlq42SQC9x6ewfVnK3je+iFMQ4eX8QTz582Er5JiLQFf3PeQpj280lenwiSuxU1gWwiaFe4bddoGKblvCCXBqyr/JXS/O2vPV5oE7Ce6EdGnT+5PX+zVz4gFXwd5R8nQVsF2o1THWvCV9k+HpFU/gL/9tYH8G8+/518wle2k7bZPubFLBb5Gv8JqfzV332sgj8RvY+I9hFRTERz2va3EdG9RPSA/P0W7bXL5Pb9RPQJWm3ji2EaoNaFfeF0P/WPy1CB4uDxRXzhvkMjHVubKXj+pdU+KNg+ruUUTZTyT4P/ZCd3kYm1WnzVs7/je1gOY+w/cgqnlsNccFeriIWWeQBmwld9T/W9un4S/FU+obNKJZ0uhh3NgwDeA+DrxvbnAfyIEOKVAD4A4A+1124CcD2AC+XP1UOOgWFeNF6Qdk8vinMtBFzEQmD77AQ2TAR44OCJUQ+vtYRxbFHK7oSvaft4JQni/HuV8k9sn02m8pfXj0gIRHGWH1jshTh0fAm9KM4Hf6X8LaWgZsJXHVf3/GOB8Qz+QoiHhRCPWLZ/WwihZM4+AJNENEFEZwPYKIS4SySy6RYA1wwzBoYZJf/fXzyE/3Dbg+lzZfsAwLFTPdtbcqi2Bh2/3iSlccVs70BEcFVu2hK+gV83+Evl31G2T+C0feK0cyfhdC9CFAuEkciVeirbJ9ciwpHwVa+paiV1AVqSrSM642T71OS9AL4thFgGsBPAAe21A3KbFSK6noj2EtHeI0eOjHiYDFPkgQMn8I1Hn0+fH1/IEr3HFuoE/0S11q1WGVeKjd3cJbDREAnfKE58/MnAR8cnTHXM4C/3E0Lum1fk/ShOx5W3fcqVf6AFf135A8BSP3lvt2XKv3KSFxHdCcDWeegGIcRtFe+9BMAvA3i72mTZzfkXFULcDOBmAJibm1u//3OYVSMSAgePL0IIASLKK/8aSV9VUeIRK39XaaRt30LCt7btI1LlPzMRwPfyk65UnkaIbJF2fU5AP8qCt08E8rLt+mcAbuWv8gMq76CWa+wG7UpvVgZ/IcRVgxyYiHYBuBXAdUKIx+TmAwD0+cy7AKzfLBjTeqI4KdV7/lQP22cncHxRU/41yj1VieN6V/5mY7eyi2EU5ytpgPp1/iqgTwQeZrqBbLWcva5/ZhjF8IgKyl/3/FWLidBW7eNI+EYF5S9tn5Yp/5GMhog2A/gigI8JIf5GbRdCHAZwkoheK6t8rgNQevfAMKuJChaHjifrzR5f6OGsjcl6qnWUf1JOqBKWoxtn2yk2ditP+JrBv77nnwT/Ky86E+985Y6kFXRuJa9sX2VFTQSG7WO0d1D76p8BOBK+2sxhFfwXZfBvm+0zbKnnu4noAIDXAfgiEd0hX/oXAC4A8O+I6D75oxqhfAjApwDsB/AYgNuHGQPDjBJ1C39QBv8XTvdxzpZpdHyqpfzjnPJfv9FfX8YRqLB9RLHU02uwkpfvEX78Nbtxw/91ceFz9AtOz6L8Q832Ucs4JtvtM4DT70NZYjhV/n7e9mlbwneoxm5CiFuRWDvm9l8C8EuO9+wF8IphPpdhXixUsDj4ggz+Cz3s2jKNrTPdespfJiCTRmYjHWqrCc1JXiXnw1bqWaels2th9Xw/f31MccHz72kJ31x7B0u1j0v5K8Ew1sqfYcYdfZIWkFT7bJnuYMt0t57yl7NIvZLqlvVAsbEbnJPkrAlfzyuspWt7X3LsfFC2BW4gs32Uut8+m9h5qi5fLeOY7Fv0/N2lnirhK0s915PnzzDjgrrFT22fhR62zHSxbUM3V/njfD8nfAEUG7uVTdqyJ3xRef7SKh3fPZlMf9yP4rSxGwBcsH0DgKwuX80BAOy9fTxbwleINDlsJny7LbN92jUahmkZSq0ffGERi70Iy2GMTVNK+dcI/iIJImW9bNYDZmM3r7TU05LwrdHSOU3UakE5MJS/foi+tH1+YMcsXnbWLC49dzOArC4/0Ov8a/T2Ua/pjd2AzPbp+PnvtNpw8GeYElTAOXRiMVX6W6a70vOvDv5Jk7L6jcnGFbOxm19W7SMGK/UMLUHZI4IQ2UU8r/wTS+7Cs2Zxx798I86cnQSQJWh15e9a41cRWIJ/6vn32jnJq12jYZiWoQTf8YV+Wu65ZbqDrTNdnFjsV3bqzNk+61755xu7NVH+fo3GbvoiLIpAs2MAm+2T7as8+WWL529T/p5xkVGvmcpf2Uhs+zDMGiKKRbrY9r5D8wCAzVL5A/l2D9b3pwlf9vx126N0MRfDIgLUnVP5Z9gUuacpciBf598L8xcZNT7l0ftatU/VDF81HyDX3kFeOJZ6nPBlmDVHJATO2ToFAGlXzi0znTT4V1k/ep3/em/vUCjBbFDtU6elc1XPHSBfYWQuGqOCs1LqeiVQdcJX3SFoyr9jev7tCrftGg3DtIw4FrjorFlMdXzcdl+ybu+W6W66JN/8UrXy9z2q3ZtmXCks4F5a7VNU/mUJYv19gL0EM7Qo/34UQ8/BpsFfT/imXT0rEr7yQhALi/LnhC/DtJu/evhZ/PpXvpfbFgmBrTNd3HzdZSDZl3DTVKegKF3EcpKX56HStigbx1omigWEQM7z90oS4LaEb9Ag+NtKMNVn6ccIjU6jKtAv97WEr6fq/MsTvulFJspaOndlnf9iP0LX91Z9wXYTDv4MI7lj3zP4zN89ndumZui+4cLt+NQH5vDBK8/HZMfPlfaVoWacNkn43rHvGfz3e56u3nGNoFRzoClfj/IqXMed8K2p/P2i7ZMp/2J7B4Wp1JOErwrqWn+gqt4+ctdshm/cOtUPDNnegWHGiShGwZePNQvijRdtxxsv2g4AtYO/ung0SfhGcfVx1xIuO8bt+VtKPWucv9Cq/L3cGGx1/oogTfjG6RgDy2IuYYm9lKwTkF+zd6kXta6vD8DKn2FSojjOKTwg8+xN9P/sZST9/MuXLbSNY5yCv6v+3r2YS1xs71Cjq2ec2jF6Sak8pqXU07R9rAnf1PapuYxjHBcWc1G2T9to34gYZpWIRNGKiOO8klSk/9lr9JtpmvCNxHgpf1UpU1/5Fy+4dTz/MK3zz7alyj/Kgr+yYHpRnPPhO6bt45GcnW2v9rFP8spe14N/2yp9AA7+DJMSa2V6CrMnjSKd1FOh5tM6/wa9feJYVJY1riUyz99YzMVxPszOnEBi+1R5/lkffrfyF9qxi9U+RdtHjbuvV/uIor2UTfLKlP+EdjFp2wQvgIM/w6SEht2iWgSb7YWB/KSeMmJN+de1fcxxrHVsSjlZzMW+f2hN+CahqqxFRmYvFd+nfPhYiNTKEcZFpqD8ZUDveJRT/rHFxsr+PRSV/3JLE74c/BlGEsV5Ja/ijOk/69sqE74imbDUpKtnFKNS5a4lQkvbhbIunbEl4WtLvJpk9fdZWCtW++QXVcm3d5DKX7Z38DTlr+eC9DV+s++TzQcwWzr3opiVP8O0GZVoVbNAI4uSVKT141UJ31h29fTcStc2Dr0Z2VrHVoLplXj+oSXh69W42Nr6+Zvvi0VxIXlF2ttHKn+1X8cn9OO88icyEr6WSV6doHjsNtG+ETHMKqHu7OP0d7GqQ6FP6ik9pmxVUCdhaY5jXBrBKc8/58UTlSzmUjznZoM2+/ssdozZ2ycuriiW7mtJ+CbHKCp/Mw9kTvLSVwEDOPgzTKtRvrAKVjYlqWiU8PWa1vnnP3+tkyphw/N32j7CHVzLqquq+uwnx4axoljR9km7eqa2T97zV0l82/iU8vfl31wxwbYPw7SXTB3K56IYTBTNEr6JdVS/zj/Zb1x8f5vnr2wwm/oPI/sMX6BC+Vv+XsXgL3L2k76vbYYvkKh20/axlaICsrGbnD+gzzdg5c8wLSYLurIyxDJjVDHahK/MOYzJiu82zz/zyIv7W0s9tYSq+3Pyih1wJHw9e8I3MBq7pQlfjwq2j63xnPquqfLXoitX+zBMiykof4uNoKib8I1iAZLtHZor//Go9c9KMIv197ZzYiv1rNNIT8Vnm6evLuTCUP76x2TVPpHRq9/LNXYrU/5RLFLbipU/w6wRzKAbrUDCV+/n31j5j43tk5xP3fOnkjsn26xqc1EWG0r559S8pbGbs9pHq//P9/nPryVgS/gWlb+XU/5c6skwLUYF+7QPjPz/bk341lX+Ikv+1a/2yY9jrVOWiHUpfzO4NlL+flH5Z719jJnGxl2CGldhEXhd+VsSvrk1fKOi8h+73j5E9D4i2kdEMRHNWV7fTUSniOhfa9suI6IHiGg/EX2C2tbkmlm3qP/gqfJOE4jFfU1F6UKpWL9Bnb8aR9VdxVoh7YJp8fzNYO6aVW0uymL/HLfyt/X20cdh7l+0fTTlH1k8f8rGpzz/fCXRmAV/AA8CeA+Arzte/3UAtxvbbgJwPYAL5c/VQ46BYVYEpUL1mnDAnvCtM+kIUMq/fMHyqnGsdWwzbzMvPr+v+squUs+yiW9ZV0/bzNvsb6oHYjOIK4VetH3ypZ4uzz/pDxUXEr5jZ/sIIR4WQjxie42IrgHwOIB92razAWwUQtwlkhqvWwBcM8wYGGalaJLwDWoEI3UMX/XzXwelnou9CCcW80tbKtWc7+2T/DZtn9BSsaO/t1T5W9tI5O2lWBR7DOU+x7cof2OSly3hq19kIvkZ6zLhS0QzAD4K4BeNl3YCOKA9PyC3Mcyq40r4llX7lAUjfcWnsl42rnGsReX/i/9zH378d+7ObbOWejrq9tM8SyG45hdlsRFb/l62hK+t6kjRcSh/vdrHVuqZVHQlx7cq/xaWelau5EVEdwLYYXnpBiHEbY63/SKAXxdCnDIsfdsZcP41ieh6JBYRdu/eXTVUhhkKc8GPMttH/89edTyfCPEACd+1WOq579A89h2ax6nlEBsmkvBiW/lKnVPzzilV/oUZtMnvsnNoWzTG7AYqRL7m3swtqOCfT/h6ub9FLOdumCT7iTQn0HblXxn8hRBXDXDc1wD4h0T0KwA2A4iJaAnAnwHYpe23C8Chks++GcDNADA3N7f2ZBCzplBJwdCwXWz9/IHqdWX1FZ88kQU7W+mobRxrMPbjqaOnAQAPH57H5Xu2ArB7/lXKv5jwTd5b504rF/ypqPxd7R2A7MJgtmvWk++2hG8yZlntEydzCfRdxs7zdyGEeIMQYo8QYg+A3wDwn4QQvymEOAzgJBG9Vlb5XAfAdffAMC8qkZFotS3Xp1O2IAmQrRerV37U8f3brvwXexGuv2Uv9j93Krf9+EIP80shAGDfwRPpdrvyT36bpy+yJG3157WUv95qOW3Doffzd3v+qfLP2T75xVxix9Kegedldf5EIMpKR9uo/Ict9Xw3ER0A8DoAXySiO2q87UMAPgVgP4DHUKwGYphVwQz6sSgGE52qTp267VNnkpJrHG3jwUMn8OWHnsW3nnoht/2powvp432H5gEAC70wTZbmZ9aW2z7mBbdRS2drSWny3GwdUegeagn+Zp2/bZnJZIyZ8jfnC7RxAfdK26cMIcStAG6t2OcXjOd7AbximM9lmFFgBt2yah8gCRx1bR9f2T51lH/Lq32U4u8bdyZPHUuC/46Nk9h3aB633PUk/vPt38VH3nohgPKGawp1yILyr9FIz9aFVV9YXTWRK7N9ulbbx6tM+Kr9UtvH0y4iUTsTvu27HDHMKhE6lL/L9gm88n49qQdN9RvB2cbRNlTwNyehPS39/qtfsQOPPncSv/m/9mOhF+HwiSUAyCVAXZ6/O+FLuddt2Kqz8j13km3dXNVR/hiBJeFrtndwJXw9uc5wTvnL3+vG82eYtUhcUP7JdpftU5nw1YKRa1JT2Thar/wjQ/kfXcCZsxO47Nwt6EcCz51cBgAcPd0DkA/KqgrQbOnsTPhS9Z1T5GgdDSANyoC7vQOQJXxzi7x4Xq2Eb+AlOaAwjtM7lbH1/BlmnDAVd2bb2PevSvjm6vxlrKiT8M3G0c6Ebxb889/lqWMLOHfbNC55yUYAwOxk4iofPZVcBGxtFYzrhzPhW6eRXmTJ0ZjdNgF7mwmFCtL5Vbgod6FzJXyVGLApfw7+DNNi0rYKRr3/SiR8XR536ThaGPsXeiEOHl8EgNysVwB4+ugCdm+dwZ5tM/i/X7sbv/AjlwAAjp4qKn9X3X7kSPjW9fwLa+tq9pK67rq6egL2SV6B2d7B4fn70gZU1T768dto+wyV8GWYcWKQhG+dBGSuzr9BwreNyv/xI6fTx/rqVkv9CM/ML+HcbdPwPMIvXfNKPH9K2T7J78CyiIp5Pmw9+QHUKpVNEq32O4Yo0pS/YzEXILs7Mds7RLGAEMnaDK7gH9iUvzz+2HX1ZJhxQXWTBIpdPcsSvmXBSG8JXTfhq49jFJ7/Q4fmcUx68IOg1/brVsj3ZaXPudum022bpjoAkH6efhpdwd/WmRPIgvhSP8bfPX7UOrYotqytq1001Gd1LG0mFK6EL5DZXJEr4Ss9fzXJSz8+2z4M01Ii47Ye0Kt1Biz1tCR8K7uAWsaxklz3e/fgd77x+MDvf+zIKXiUKFnd9vnbx5KAfOGZs+m2ju9hpuunzdTIWoJpT/gWG7sloepP7/0+3n/z3Xh2fqkwNpvy9zwCUb7aJ5fwNf60WVdP7bN9Nbs4GVxZwjeMY6n88/MFeBlHhmkppqer/3bZPn5FwjdX51+jWsUcxyj6+Z9a7uOUnIU7CPufO4Vzt81gopPVvvejGDd//XHMnbsFP3D2bG7/zdNdAPkkK+BeDEcF2GIQT37vO5hMHju9XPwOoaN1hrJjhCWZXMgteMr28Qrb+tqaAPZJXpTO8FXvabPn374RMcwqoAchM+Fra+wGVJd6xgMkfHPjGIHyDyMxVNuIp44uYM+26UT5y+P8j28fxMHji/jwmy+AuTbTRmn9BEbJlKvaxzW3Qr3/pAz6tvNuLtGoUFVZqfK39P5RqJm4+a6eUvnLwUaui4xP6SQvz0z4su3DMO3ErvyT507l79VT/r5Xf9lHc9GQlUTISpRh7igOHl/Ezi1TuWZnv/83T+LiszfiTS/bXth/swz+hZWvHAu4OxO+xnPbd3DOvNUSsQDg+x5UzDeDeDf1/LX3+9lcgWSM7otMGAv0o2wZyrS9Awd/hmknsS34lyzjCFQnfFPbJ5fwrT+OlU74mt1Km3JqOcSJxT52bp5G4HnoyS/z3MklvHr35oLqB7Kkb8HGqert40j4mvvp2BZZUe9V1TrJsbOg7FrGUe9AqhZ2VwluV8JXlf6eWOxj03T+ose2D8O0lNASdCNHIFJUlXrqtlGdfvTmOKIVLvQPNY9+EA7J+v6XbJ5MWh7I4/XC2KlsN8sgaHr+VS2dXfsrzAlmAHL19eZ79YSvpzXaK9T5BypRm21Llb+q9nG2dCbML/WxHMbYNtPNHZ+VP8O0FF2BxobtY/rVispJXlrC2FXaWDaOlVf+mWc9CAdfSIL/ri1TCDTPvxfFTmW7yeH5u7p0upS/eedg+w5xLHIdPRW+XGQl1pS/OlyhvYPlomBW+9jW8FVjfHY+mdNwxoaJ3HHa6PnzJC+GgV35Z+0Z7O9RHq8LPXmpuno2Uv4rHfxT5T/YcQ9I5b9z83TS4147nquUcaPD81fPzWuhbSlG23NzdjHgVv6q5446Nmk2nKu9Q972Mer8Y/vcD9+jdGLbtg2G8g+41JNhWokeaM02D4MnfOV+ej//CuUfjVD591PlP5jtc/CFRXR8wpmzE2m/G1XdUmn7FDx/yLEMmPC1nBuXIldVWerU522f/L6BNeGrqn0yO9CW8PU9Sj9jLSj/9o2IYVYBW4mlrT+8jl834etpXSkblHpW7dsUFbwGvagcOr6IHZsm4XmULnCi8gdVtk+h2sfRrsGVZ1Hn7+xNk/I7FC9gLi8+8fzjnO2j9jOT1Nkyjlqdv5rhq9lmtjyQfkFIgz8lk8xcAmI14eDPMLDbLVX9/FUi0cUgdf42+2mlGDbhe/D4InZungKgFjiJs+DvUv5T3XR/HXU+zAtc5Ej4eh5h01QHrzpns/wOLuVfHEdykUYu4euyfbqWhK+q9smUv7vUU7F1Jpvc1vE9ayXUasPBn2HgKPWsUP6DJHyrbB/bOFaK/gokfHduTnr3dHxV0y7k86qEb71qH1fCFwBu/8gb8NNvusD5HZK2CsUxmMqfKFP8rjYS5gLugDbJy5XwlfttnAzSi4hHhIkWWj4AB3+GAeAq9SxX/mo6vwu9MZwKDFV2+yiVvxrrIAnffhTj2ZNL2Lk5sV06srePUv5Vnn/R9kl+m1/RlfAFgJdsnsL0hJ+Ox0TvqaOjLtKxdjFWw3Ut5qJ/vlLxakWy2JHwVResM2Yncp/dxvV7AQ7+DAPASPgatk9Zwrcs+AvN9qmr/PON3Va2zl8FzEHaOzxzYglCADu3SNvH89CLBHqhCv7l1T51J3m5Er4K04LJv1fANgx1ka5j+6TVPtr2C7ZvwMbJAN988li6UperqggAzpjJgr/nUSubugFc6skwAOxVNnWWcSxP+Gb7qb2qkrijrPZJE74DKP8DL2RlnoC0faI4neXrSvjOTgTwqGSSVyH42xdzSd9XsqiLvnC6jlqMJVfnL49v7m5bzMXzCHN7tuKeJ4/h299/AbEALjxrQ+Fz1HtUmSeQ/Ntp4wQvgJU/wwDIK3IVJLKEr/09Vcrf3t6hIviPstpHtSQe4LiqhfIOWW2TTPISlbaP5xE2TnXck7xq9vZRpDX3tmqfWFj/Vpny1+r8PbvyVxcp8/OvOG8rHj9yGp+5+2l0fMKbX35m4XMCS/DfOBWk1lfbYOXPMHAp/xqlnnWqfTyCQH7uQJNxrBT9VPk3t32W+hEAYLqbeO4dL6nz74fJMcvq2DdPdSyN3dQkr3qlngqz5j73XiHQ9fzie9LePkiPbXbdVHQtyh8ALt+zFQBw630HceVF27FxshjQVb5BlXkCwEevfjkW5blrGxz8GQZ2z79OP/961T6AQE3lP8JqH3W8QS4qpsJPEr4itX3Kkppze7bmAiJQsoB71TmX223fwdXPP+vto9k+jvYO6QpcxsXnlTs3YbLjYakf4+pLdjjGlvzepn3Xbcb3bhMc/BkGdsUdC5ErCzSpq/wT26f4OXXHsVKkCd+Bqn1USae0ReQM3+yi4E5q/tr7/l5hm7JniqWe5cG/Y5RdAsDdjx/F//zOodKZt70wzid8lefvSPiax+kGHl51zmbc88QxXHXxWdaxKWvrjJmu9fW2MZTnT0TvI6J9RBQT0Zzx2g8S0V3y9QeIaFJuv0w+309En6A2zn5g1h02xR05esUofI9KG7Xl6vzVpKZG1T7tmeFrU/51Jnm5cM14rlNhBeS/w9e+dwSf+bun8dTRBatdpNo76HX+XkW1j+0O4kNvugAfvfrlhbsYha3Us80Mm/B9EMB7AHxd30hEAYA/AvBTQohLALwJQF++fBOA6wFcKH+uHnIMDDM0toSva8UmRdVKXsMmfEfV1XOQUk81Fj0hWifh68LV5bSqwspW6tmX5aYnl0Kn8o9Fvs5ffX6x2see8AWAKy/ajg9e+VLnd1LnZtt6UP5CiIeFEI9YXno7gPuFEN+R+x0VQkREdDaAjUKIu0SS6bkFwDXDjIFhVoIoKgbdOsq/bsK3bnsHfRwrX+c/eKlnWs8vo2WgPP+wfIavC9eC9lW9gjyP4FH+AqZP+HKu5BUJq+1T6OdfovyrUBeUNvv8OqMq9bwIgCCiO4joW0T0c3L7TgAHtP0OyG1WiOh6ItpLRHuPHDkyoqEyjL3EMnKsCauoTvjK/ZrYPiNcwzdL+A6i/BM/XX2Prk/o5er8mwVL33E++lFc2QgtkP350/doj12N3WJh2D6OUs+OI+Fbh3O3TWP31mlsnFwbqdTKURLRnQBs6e0bhBC3lRz39QAuB7AA4K+I6F4A85Z9nf/ChRA3A7gZAObm5lb2fwLDaFgTvlW2j1+zvUMu4VtvHJ2KYw/CsAlffaKWKrlc6iVljF2/WGJZhqvapxe5VwXLPptyCV9l+wDu4J+f5JX9PYrtHYq9feryj67YjWsvP6eVTdxsVAZ/IcRVAxz3AICvCSGeBwAi+ksAlyLJA+zS9tsF4NAAx2eYFcVa6ulo4KXwqTzhq3vMTev8JwJ/pGv4CiEaBam+EZTVhWChFwJovliJawH3figqk8eBR7n+RFW2j+95hTp/9ySvwYM/4K4MayOjsn3uAPCDRDQtk79XAnhICHEYwEkieq2s8rkOgOvugWEac/TUMi7/+J148OCJRu9ztXdwTTYC6id8fS3hW7e9QzfwRlDtkwXJpsc2g7/y/hfkBKaBE76xwH/6y4fxs5+7T/uc8gAa+Plz048E9mybxkzXx4QlV5A2dtPq/F1dPac6yR3MZKfZncxaZChziojeDeC/AdgO4ItEdJ8Q4h1CiBeI6EYA30Ri6/ylEOKL8m0fAvAHAKYA3C5/GGZFOHxiCUdOLuOxI6fwip2bar8vDbq+l6rz2NEiWKFWbnLZQ3p7CL/hJC99gfSVom8ktYMG8a0f5pdqVI8Xe4MF/9T2EQIPH57HkZPLcow1bB+PcnmLXhRjqhvg9/+fv5cu9qJjNnbTl3E0/25bZ7r41HVzeM35Wxt9n7XIUMFfCHErgFsdr/0REpvH3L4XwCuG+VyGcaESkE3bFucUt1q0Q1RU+2gBzENxP73UUzhsDtd7JgJ/5ZW/FjCbWkr92LR9ksenl5XnP1i1TxwnnUGVdVPL8/fyF0Z1t3DFefaAbVP+2SSv4v6uSVzjBjd2Y8YKVZLYC5tVtCi13/Epp/yrEr6AW83rawA3rfPvBl5lfqApubUCGvb3SRZp12wfpfz7Ye55E5IqHOSqhvqRcJZ5KlRTOUUYudcQBpILjcpzAMYavmvIo19p1kZNEsPUpJ8GkWbBLbR47XUSvoA7oMea8lcyqyqgh6ny91Y+4ZtTy82OHRpevGplsNCLBl6j1qPkfOgN4vphHc+fcn/fXkWeIJAreemlt65qn/UEK39mrBhU+ce24F9jkhfgDui5Ov+6C7jnxrGyk7yGTfgGlkXNTy9H6A64Rq1HlNo+Pe2iXcf2ySd8y99jNnYra++wnuDgz4wVunfcBD3hq/fzr2rvALgDemTxmOvW+XflDNqVpG8EzEbvjUSuc6fy+Bd6YWO/X6GCcj8Saa1+Pc/fK5R6lo1BfY7N9mHlzzBjwvKgnn+quP006NZV/i57Jo5FWlaoYkzdOv9Rl3o2TvhGMbqWSV4LvWjgNWqTeRLJ32pZU/5VF5Nk8Xh9kle55x/IFddy7R0cpZ7rCQ7+zFihFKFN2e598hi++shz1vfZEq2RY6FuRR3lr/ZRF4DKOn8xymqfwfsGuWyfxV408Bq1RMndVU92BxVCyDuM8uOZPZX6UVx6AfJKqn3Ws+3DCV9mrCjz/D/51cdw+MQS3vSy4hJ8aYmlNoEoFhV1/lRH+WfBpWrN39w4RlHtM0TCtx8JTHVtk7zCgdeozWyfGEIg7RI6W9EbJ5DtpNOxxXG6vKN1/7S9Q/KciKD+LK4lOtcD6/irM+NIWbXPUj/CcmhfUs9mt1TZPq7OlIrYqBZSCc4y9OC/0p6/bpUMNMPX022fTPkP5fkLkV6o+1GMXjhAwrfC9lGT8bIlIln5Axz8mTGjF7oTvr0wduYClHrXG6pV9fNXHT9dE7eiOB9cqlpA58ex8p6/qx9OHcxa+o42yWtQ5a8uhukFOxS1PP9E+Zu2T3VJrnqP7vmXte8Ydzj4M2OFCvqqz7zOchinCWETlZzVA/TQCV+jWsinatsnaSlB6cSklWTYhG9ga+/QjyonZbnwSC0Ik4xlOYrkZLLygNyxtHcoVf7p0o9c7aPDwZ8ZK8psnyrlH3hePvhXTfKqSvjGhu3jVds+oXyPmpg0DHufPJZrcKeXeja1lPpxXpHryd9BE76+R7mLcT8Ster8fUt7h7K7BXWH1pOWn+fJZnvrOPADHPyZMaMs4bscuj3/RKXnk7JxXG+Gr0tFR6J5wjeW/YSqOobW4T/+xUO48Svfy8aTa+zWsNontLd3SB4PqPy9JA+jUD1+qkpHO0Z7B7P1ROFzLLaPugCsZzj4M2PFwMo/Usq/QXuHqoSv0RU06S5ZPv5kHInyr7pLqGKhF6VdNwGjsVvjap/YupgL4F5ysQqfKBf8kwXha/Tz1xZziWOBKM4vNFPYXyn/SCV8Kb0ArGfW+ddnxo3ShG8UIxb2pmaxkJ4/ZcHcLNU0cS1FqDBzBr5Xo72DzBP4/vDK37zT6UeiMk/hotDPfyWUPxGW+5p3r5R/hY2k3xX14+oF5NWdhLKYVG6HlT/DjBE9qWitto8MNLakb6Iemyl/ryKQRgMkfKM4U/7DVvss9WMs9fUkb4xJGQgH6+ppD/iDev6eR1jSLk5qslfVxaTjZWWwysopu1uYkAsXqLsMIsKGiQAzE+t7mtP6/vbM2FGm/JejLB8wM5F/LZQq3/egVfuUlwIGFQlfM2dQN+HrSVU6yHKLOsv9ovKf6vo43YsaK//Q7OfvDa/8faKcLbXcjyv9ewC5uyLVE6jsAjTZyfoQAYnyv/6N5+OaV+8caNzjAgd/Zqxwef5Cm0xkU/6xVNy+12Alr8qEr6XOv0appxoHAMQCGFBYF0pbo1ikKrhJwjdtu2BZzAUY3PP3PMLCcpg+V2sDVB1PL/VUf+eyJLH6zou9zPPfPN3F5unuQOMeF9j2YcYKV7WPfidgs4RUiWVO+Q9Z6mkuBuNTvUleHlGawGxalaMQQhSCfxjFqQpu0t5BnwCn0B8POsPXI+RsKbUqWJ01fJXto/6uZXcL6jsr22c9T+zS4eDPjBWZ8s8HNz0I2so9VSuGQPP86yZ83f388wlfr4aPr8ZRVUlUhQqKy/2i7dP0uH1LgHU9boLv5at9Ti+rVcHqr+EbNvD8U9uHox4ADv7MmNGLHMo/F/zdyt+j+pO8miZ86yRxQy3hCwwe/NV3XNKVfxxjqiNtnwYJX7XKVrDCnr9nlHqerrkYfKAtbq8uTGWlnpnnz8pfh4M/M1a4Er7LFcE/a++A2u0d6iV8s+ceUeUC7soqGlb5q6DaC+N0EZMwEpiUwb+J7aPKKfV+/kTZBaqqBbML36PcOJTyr6zz97w0GV7H9kk9f7Z9cnDwZ8YKV8JXV/52zz8uTPKqWsDdq0z4mnX+dZR/nFP+g9b66/Xz6mIXxlnwH8T2CYwAq9T2wF09jSB8WtoyVRcT/a6oTqmnUv6qsohjfwIHf2asqGP72MpA1cItvpd5+GbwNlHBr6y3D5mef0XMVeWlVe2iq7Dd6SQJX6n8GySSlcViqmv1fFDbxzy1tT1/+brq/1/1nszzZ+Wvw8GfGSuUP20qfz3JqydBFZFU3Er5CyEqWzqrC4Mr4Wv28/frrOQVJ20Uhlb++veVj/uxSCd5RQ1sn8xayZ8LFXAHbu9gnNuF5Zqev3ZumtT5Z9U+Aw137ODgz4wVPa3aRw+0lcpfIJ1cBST19VFVnX+FOi+2d6i2fSIh+817zYO0Tk75923Kv0Gpp0P5p57/ENU+Oqfqev5pi+Y4uzDVqPNn5Z9nqOBPRO8jon1EFBPRnLa9Q0SfJqIHiOhhIvqY9tplcvt+IvoEDTp9kWEs6EFetzZswVBHKX+9vr5uP393Y7d8WaFXq71D3vMfdClHvYpGKf8wEuldRaNqH4e1kir/Qds7GOd2oXa1TzZXIb0wldRvTsgLw2KfPX+dYZX/gwDeA+Drxvb3AZgQQrwSwGUAPkhEe+RrNwG4HsCF8ufqIcfAMCmuyVzVnn8S6FVAimM5u7YkqNRK+Hp55V9nGcd8tc9gk7z0i52aSNWXLRrq3IHo9BzllOr54KWe+edpwrdqklcu4auUf3lJrrqbIMLA7TLGjaGCvxDiYSHEI7aXAMwQUQBgCkAPwDwRnQ1goxDiLpHUn90C4JphxsAwOrmFvTXLpMrzj2Oki6gASeCusn3qJHwb9/OXSz+OotpHNY3rGMsgVuGaSDVswte0fdKEb0UOQZ2bvm77VIxhQvr+672Tp86oPP8/BXAawGEATwP4NSHEMQA7ARzQ9jsgt1khouuJaC8R7T1y5MiIhsqME70wxoycxeqa2GVT/mEcp8snAonXbk7SMqlK+IoBFnAPZcI3nUA2sOeft31Ufx5lbTVpG+GyfbI6/8EneSlmun7a3qHK8+/kqn2qSz2BzPdnvz+jsrEbEd0JYIflpRuEELc53nYFgAjASwBsAfANeRzbmXf+6xZC3AzgZgCYm5tb2QVNmbGkH8XYMt3F6V6UuwuoqvPPEr7qeZIwHsbzt9b5V3n+MuG7UjN81WN1nMD3ZHuEQer8HdU+QyzjqJjqBprtU72MI5BYYnVKPYGs4odjf0Zl8BdCXDXAcX8cwJeEEH0AzxHR3wCYA/ANALu0/XYBODTA8RmmQCyV4IaJAM+dXHbO6rX385elnqmqjIdeyUvNHVDUWckrKzkd1vbJ21xhGvyT/kXNEr52dZ1O8hpS+XcDDxOBh/mTfQDVnr96Xa35W+c9KunLyj9jVLbP0wDeQgkzAF4L4LtCiMMAThLRa2WVz3UAXHcPDNMIVd2jFulopPzl5KrUyokFhCgPFlUTscw7hzoreakLRpC2dF4Z5Z8GSblIfRM7KXQpf284z1+dv67voeNTbf9eJeHDKLN9qqynzPYZaKhjybClnu8mogMAXgfgi0R0h3zptwBsQFIN9E0Avy+EuF++9iEAnwKwH8BjAG4fZgwMo1BBfWbC7fl3fa9U+afJRDlZrEz5V1kztmqfuqWe/tCef760NdKUf6fhEpGuoKwqbAZfzCX53Q283N1D1Z2EXo6rX9TKULYPK/+MoRZzEULcCuBWy/ZTSMo9be/ZC+AVw3wuw9hQKnBDifLfMBk4l3HUE769KLFNSrt6VpR6mr2B6iR8VamnCnADe/5Gnb86N0nC12uY8LXX0gcrpPw7PjVqEa3GUXeGL5Apf479GTzDlxkbVLBXto9e1bMcRgg8wlTHt/bzV8FfqfmeVP7DLOOYJHyz5/WUf1KRk11Yhq/zXw7j9DhpwncA28espVcBd/DFXDLPX1f7dRZwT8aVeP5E5RdpQFP+7PukcPBnxgal7qe7Qe65eqwSi/ZqH1P5J/vUau9QsphL05W8VIXQSlT7qJLXpX6UBvus1HO4xVySY0nlP2hLZ8pso0bKX7N9enJ5yaqJW0r5c51/Bgd/ZmxQAXvDRLFnfS+KMSEVpjX4R3nlrwJemfInIhDVT/jWWcBdjWPYap+lfoSZiQBERsLX9xB4XqHxXRlO22fYGb5awnciqB/8066eUvnXufPISj05+Cs4+DNjQ5bwLSr/5X6m/K2ev8i3d1BecpWdULY6VyHhW6e3j1pOsmL2cBXLYdLETX3fKFfq2ay9g6uFQjet8x90hq88TpApf6+GhaPPfg6juNImArjaxwYHf2Zs6KfK35LwjWJMBL5b+af9/JPosBzVC/5eiZVTqPP36tT5q1LPIev8wwgTgYeJwMdyX0/4erllEOuQzhFwKP9h6/w7vpdeQOrcReS7eorCIjM2uNqnCAd/ZmwoVf5hJJW/K+GbL/VU760KFmUqOi7M8K2u29fXFUieD97bZ6LjYbLj5RO+cg5Bk8Vceo6KmqBBwLaRJnx9L63Tr3MXkVpzcX3bZ6LDyt+Egz8zNvRKqn16oeb5u7p66gnfmraPV1LBo46pqJXwlc3ghlf+MSYDHxOBjyVd+fuJpdTU9un4VPDLO1qp5iCoc9MJNOVf4y4iSC+McTq2KtQiNuz5Z3DwZ8aGtJbfMckr9fyt/fztCd+q6pBS5W909ayV8FWlnkO2dF7qR5joZDmOsJDwbWb7mJYPMLzyV8G/63voBvUvJIHR3qHO56fKnyNeCp8KZmxQAW2mW/T8l8NYBhmH8lelntRM+Zf1xk+Oqe3bJOG7Asp/IvAxkdo+WqmnR40uKr3Qrq5Xag3fbkDNPH/NEuuFol7w594+BTj4M2NDmeffC2NMdHx0fVfCN19l01+RhK9R51+jykaNo6ppXBW5hG+oN3YbJOFrV9fnnTGNPdumK8+RC1/3/BtUDukJ3zCOa1lFmefPwV8xVHsHhmkTKmBPd/3ccyBT/koJm5greal9qmaENkn4eh7VSPjmlf8wk7wmApnw7We2T7qYS5P2Dg51/f7Ld+P9l+8eaHyA5vlrCd86Kl7NN+indf71PX9O+Gaw8mfGBqXouzKBuJxL+CYeeNf3C8pfCCGXbNQ9f9nYrUIpViV8cyt5VSR89XEMrfz70vYJfDnJK0v4+h41Whi+H8cDz+ItIy31zCV8qz/H1/oe9UN7PsKElX8RDv7M2KC8fOXtq86cgFTCqfLPl3qqAKsH3czzL/9Ml/JXgdys849F8pqNdByUtXQeeIZvmCV8l/pRWurZke2T+42qfURl18xByCd8m3j+qtQzWcaxlu3Dnn8BDv7M2KBsnmTGKBW6eibKP6l00atuQi34pzN8a7R3ANTErWIgVZt8Q/kDbjWvxuF5lFalDKP89Rm+em+fpJ9/E9unXkVNU9R1sRs0nORlNHarZft0uKunCQd/ZmzIJiMVe/jonj+QnwOgfPhBEr4uKye7m9D2VQHdofzVONRELGCwfv5CCGfCV5V6NrmjUOsKrzTmYi7qcRV636PapZ6s/Atw8GfGhrzy9yzK30+Di5701e2WpglfVwWPCuSm7QMArlxrlLsDUdua1/mHcWI5TQRZgltfjavTsNpHdc5cafSunl3Ze6dOnT+R+g7JHU2dsU1ynX8BPhXM2KCUfuARuoGR8JVtAJQC7NmC/wCTvHxHBU96N2Gzfao8fy+ZTTs7GeD4Yr/0822oC9dE4GOy42O5H6cevy9bRzSxk+o2T2tK6vkHmfKve5FRF90eK/+B4eDPjA29SKAbJL3du76XduYMo6SrpertAyCX9C1P+FaXetosFP2YiuoF3/PvOXfbNJ46ulD6+TbUKl5pwjeMshm+nkr4NmnpPBrPn1LlT1nCt2aTuI6cpVy7vUPq+XPwV3DwZ8aGXpg1+dJn8qrfqreP2ldhC/5DJ3zl4XPtHai8TXMh+G+dwdPHmgf/pTD7vhOBByGAhV5yQUhaOifb6qr//shsH6TjbNoeOlmQJq49NqX8R3ADs2bh4M+MDf0ozpUMqgCuevmo3j6A4fkLi/IfNuErSpS/y/YxrKLd26Zx4IWFxhU/Svkn1T6J4j29HAKQCV9tJaw61FXXTdEneWV/t3qf48ukdd1KpEmu8y/AwZ8ZG/QeNB2fUnWfKX/fqvxV8nOQOn9XwjeKiwlf3ytX/vo4AODcrdPoRwKHji+WD8JgWVf+sropVf5636CaSd9R2T6eFvyb9glSCd+kzr86oLPnX4SDPzM26Mq/G/joyeCmK/+uRfnryVnfrPZZyYRvhfKPjbuF3dumAaCx759L+Erlf0oq/9wSkTWDf1hzwZSm2BZwb5LwDSOBMBb1+vmnLZ0HHOwYwsGfGRuWNYXazSl/mQDVEr455a8tceg3rfOvTPhq+9ac5JUlfGcAAE8dO106BpM04asp/9PLIQJZRaTOUV3bpzcq28eygHvdVcE6sn1HFNfz/APfS1plc/RP4eDPjA19I+GrAviSVfln1T7KhvE05V93JS/f0aM/0o6pqKrzj43gv2PjJLq+h6cbKv804dvJchynlsPU6888//rKf9B1estQ52NCy8XUvcgEHmFJWll17xYmOz7X+WvwqWDGBlfCV6/2sdX5h9Zqn+xuoAxXqadp4SSPk98u2ycdh7xg+B5h19ap5rZPqvyzhO/TxxYwJZOeTdcK6EejmeGrzscgnr/vERb7KvjXG9tE4LHy1xgq+BPRrxLRd4nofiK6lYg2a699jIj2E9EjRPQObftlRPSAfO0TxIW3zAqhJnIByPXtz3X7VME/qlfqWdnVs7K9Q7HUs26dP5AkfZ9qWO6pPP9JTfk/dXQB73rVTgDQWkc0qfZpl+ff8b00id1E+XO4yRj2L/oVAK8QQvwggO8B+BgAENHFAK4FcAmAqwF8koh8+Z6bAFwP4EL5c/WQY2AYAPm+8x3N9tEToGmpZ7884dtbqfYOloSvq6e/7W7h3G0zeProaWcnUBu579vJ2iZ88MrzAeSXQazDqOr805bOsuUE0KzOf7Fh8E+U/wADHVOGWsxFCPFl7endAP6hfPwuAJ8VQiwDeIKI9gO4goieBLBRCHEXABDRLQCuAXD7MOMo4yc//c2BZkkya4+njy3gNedvA5AEkWOne3jbjV9La9z1SV6/+uVH8DvfeBwAUvtALeBOhHRyVZ32Dk8ePY233fi13PZlywxhdayf/PTe9CKko8ahX3B2b53G6V6Eq278Wm3L4oRsCaHPa3jvpbtw9qYpAJny/4k/+KZ1HLZxBSOImuqYg9T5Bx7h8edPJY/r2j4dv/LvuZ5YyZW8fgLAn8jHO5FcDBQH5La+fGxut0JE1yO5S8Du3YOtGLR760ztCgJmbXPhWRtSa+NHX/USHDm1nCrmK6c6uPCsDej6Hv7Z68/D4RP52vnXnLcNr969GQDwkbdeiO89exJbZ7rYtWWq9DN/bO4cp5KfO3cLLt+zNXu+Zyvec+lOLPUj6/4A8Nrzt+HV52xOn7/9krNw/4Hj1nWHyzhr4yTOnJ3AlukufvL15+Gfv/H89LUrztuK97x6J5ZC9zh0Ltoxi3e+8uxGn1+HH3rpGfipK1+Kl++Yhe8RfuaqC/G2S3bUeu8/ed0ebJ89jK7v4fUXnFHrPR9+80sxO9kZZshjBVXdThLRnQBsf5EbhBC3yX1uADAH4D1CCEFEvwXgLiHEH8nXfxfAXwJ4GsD/L4S4Sm5/A4CfE0L8SNVA5+bmxN69e+t/M4ZhGAZEdK8QYs7cXqn8VaAuOfAHAPwDAG8V2ZXkAIBztN12ATgkt++ybGcYhmFeRIat9rkawEcB/KgQQjfWvwDgWiKaIKLzkCR27xFCHAZwkoheK6t8rgNw2zBjYBiGYZozrOf/mwAmAHxFllDdLYT4KSHEPiL6HICHAIQAPiyEUAbjhwD8AYApJInekSV7GYZhGDvDVvtcUPLaxwF83LJ9L4BXDPO5DMMwzHBwGQzDMMw6hIM/wzDMOoSDP8MwzDqEgz/DMMw6pHKSV1sgoiMAnhrw7WcAeH4Fh7PStH18QPvH2PbxAe0fY9vHB7R/jG0c37lCiO3mxjUT/IeBiPbaZri1hbaPD2j/GNs+PqD9Y2z7+ID2j7Ht49Nh24dhGGYdwsGfYRhmHbJegv/Nqz2ACto+PqD9Y2z7+ID2j7Ht4wPaP8a2jy9lXXj+DMMwTJ71ovwZhmEYDQ7+DMMw65CxDv5EdLVcQH4/Ef38ao8HAIjoHCL630T0MBHtI6KPyO2/QEQHieg++fPOVRzjk0T0gBzHXrltKxF9hYgelb+3rOL4Xqadp/uIaJ6IfmY1zyER/R4RPUdED2rbnOeMiD4m/10+QkTvWMUx/ioRfZeI7ieiW4los9y+h4gWtXP526s0PufftEXn8E+08T1JRPfJ7S/6OWyEEGIsfwD4AB4DcD6ALoDvALi4BeM6G8Cl8vEskoXvLwbwCwD+9WqPT47rSQBnGNt+BcDPy8c/D+CXV3uc2t/5GQDnruY5BPBGAJcCeLDqnMm/93eQtEM/T/479VdpjG8HEMjHv6yNcY++3yqeQ+vftE3n0Hj9vwD496t1Dpv8jLPyvwLAfiHE40KIHoDPIllYflURQhwWQnxLPj4J4GGUrGPcIt4F4NPy8acBXLN6Q8nxVgCPCSEGnf29Igghvg7gmLHZdc7eBeCzQohlIcQTAPYj+ff6oo9RCPFlIUQon96N/Ep7LyqOc+iiNedQIReo+jEAfzzqcawE4xz8dwL4vva8dLH41YCI9gB4NYC/k5v+hbz9/r3VtFUACABfJqJ7ieh6ue0skazEBvn7zFUbXZ5rkf/P1pZzCLjPWVv/bf4E8osrnUdE3yair8n1tlcL29+0jefwDQCeFUI8qm1ryzksMM7BnyzbWlPXSkQbAPwZgJ8RQswDuAnASwG8CsBhJLePq8XfF0JcCuCHAXyYiN64imNxQkRdAD8K4PNyU5vOYRmt+7dJRDcgWXXvM3LTYQC7hRCvBvCzAP47EW1chaG5/qatO4cA/hHyQqQt59DKOAd/1yLyqw4RdZAE/s8IIf4cAIQQzwohIiFEDOB38CLcwroQQhySv58DcKscy7NEdDYAyN/Prdb4NH4YwLeEEM8C7TqHEtc5a9W/TSL6AIB/AOAfC2lWSzvlqHx8LxJP/aIXe2wlf9O2ncMAwHsA/Ina1pZz6GKcg/83AVxIROdJhXgtkoXlVxXpC/4ugIeFEDdq28/Wdns3gAfN974YENEMEc2qx0gSgg8iOXcfkLt9AMBtqzE+g5zSass51HCdsy8AuJaIJojoPAAXArhnFcYHIroawEcB/KgQYkHbvp2IfPn4fDnGx1dhfK6/aWvOoeQqAN8VQhxQG9pyDp2sdsZ5lD8A3omkmuYxADes9njkmF6P5Pb0fgD3yZ93AvhDAA/I7V8AcPYqje98JFUU3wGwT503ANsA/BWAR+Xvrat8HqcBHAWwSdu2aucQyUXoMIA+ElX6z8rOGYAb5L/LRwD88CqOcT8S71z9W/xtue975d//OwC+BeBHVml8zr9pW86h3P4HAH7K2PdFP4dNfri9A8MwzDpknG0fhmEYxgEHf4ZhmHUIB3+GYZh1CAd/hmGYdQgHf4ZhmHUIB3+GYZh1CAd/hmGYdcj/ASioj+87AzxFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > -110:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
